import numpy as np
import csv
import random
from datetime import datetime
from time import strftime
from dbfpy import dbf

'''
to calculate the false nagetive, false positive for satscan

'''

def getCurTime():
    """
    get current time
    Return value of the date string format(%Y-%m-%d %H:%M:%S)
    """
    format='%Y-%m-%d %H:%M:%S'
    sdate = None
    cdate = datetime.now()
    try:
        sdate = cdate.strftime(format)
    except:
        raise ValueError
    return sdate

def build_data_list(inputCSV):
    sKey = []
    fn = inputCSV
    ra = csv.DictReader(file(fn), dialect="excel")
    
    for record in ra:
        #print record[ra.fieldnames[0]], type(record[ra.fieldnames[-1]])
        for item in ra.fieldnames:
            temp = float(record[item])
            sKey.append(temp)
    sKey = np.array(sKey)
    sKey.shape=(-1,len(ra.fieldnames))
    return sKey


def build_satscanresult_dbf(inputDBF):
    sKey = np.array([])
    fn = inputDBF
    db = dbf.Dbf(fn)
    for record in db:
        temp = float(record[db.fieldNames[2]])
        if temp < significanceLevel:
            temp_id = int(float(record[db.fieldNames[0]]))
            sKey = np.append(sKey, temp_id)
            temp_id = int(float(record[db.fieldNames[1]]))
            sKey = np.append(sKey, temp_id)
            sKey = np.append(sKey, temp)
    sKey.shape = (-1, 3)
    return sKey

def build_satscanresult(inputCSV):
    sKey = np.array([])
    fn = inputCSV
    
    ra = csv.DictReader(file(fn), dialect="excel")
    print ra.fieldnames[2]
    for record in ra:
        print "here"
        temp = float(record[ra.fieldnames[2]])
        
        if temp < 0.15:
            temp_id = int(float(record[ra.fieldnames[0]]))
            sKey =np.append(sKey, temp_id)
    return sKey

def cal_sum_within_id(list, id):
    # list: [value_1, value_2]
    #id: [id_1, id_2, ...]
    total = 0
    risk = 0
    i = 0
    for item in list:
        total += item
        if i in id:
            risk += item
        i += 1
    return [total, risk]


def findRegionID(unitMatrix, clusterID):
    # unitMatrix: [unitID, regionID, P-value]
    regionID = []
    for item in unitMatrix:
        if int(item[0]) in clusterID:
            regionID.append(int(item[1]))
            #print item[0]
    regionID = np.unique(regionID)
    #regionID = [regionID]
    return regionID

def findUnitIDforRegionID(unitMatrix, regionID):
    # unitMatrix: [unitID, regionID, P-value]
    listID = []
    for item in unitMatrix:
        if int(item[1]) in regionID:
            listID.append(int(item[0]))
    return listID

def calSum(data):
    # sum: [mixed, rural, urban, total]
    sum = [0, 0, 0, 0]
    for item in mixed:
        sum[0] += data[int(item)]
    for item in rural:
        sum[1] += data[int(item)]
    for item in urban:
        sum[2] += data[int(item)]
    sum[3] = sum[0] + sum[1] + sum[2]
    return sum

def calTPFPFN(dataMatrix, usedClusterID, unitID, pop, cancer):
    temp_result = np.zeros(12)
    #[POP_TP, POP_FP, POP_FN, DIS_TP, DIS_FP, DIS_FN, COUNT_TP, COUNT_FP, COUNT_FN,
    #   POP_TP/(FP+FN), DIS_TP/(FP+FN), COUNT_TP/(FP+FN)]
    for item in unitID:
        if int(item) in usedClusterID:
            temp_result[0] += dataMatrix[int(item), 1]
            temp_result[3] += dataMatrix[int(item), repeat + 2]
            temp_result[6] += 1
        else:
            temp_result[1] += dataMatrix[int(item), 1]
            temp_result[4] += dataMatrix[int(item), repeat + 2]
            temp_result[7] += 1
                
    temp_result[2] = pop - temp_result[0]
    temp_result[5] = cancer - temp_result[3]
    temp_result[8] = len(usedClusterID) - temp_result[6]

    temp_result[9] = temp_result[0]/(temp_result[0] + temp_result[1] + temp_result[2])
    temp_result[10] = temp_result[3]/(temp_result[3] + temp_result[4] + temp_result[5])
    temp_result[11] = temp_result[6]/(temp_result[6] + temp_result[7] + temp_result[8])
    return temp_result

def calSeperateTPFPFN(data, usedClusterID, unitID, pop):
    temp_result = [0, 0, 0, 0]
    #[TP, FP, FN, TP/(TP+FP+FN)]
    for item in unitID:
        if int(item) in usedClusterID:
            temp_result[0] += data[int(item)]
        else:
            temp_result[1] += data[int(item)]
                
    temp_result[2] = pop - temp_result[0]

    temp_result[3] = temp_result[0]/(temp_result[0] + temp_result[1] + temp_result[2])

    return temp_result

def findHighestScore(listID):
    win_score = [0, 0, 0, 0]
    win_p = -1
    #print repeat
    
    for p in range(0,3):
        if p == 0:
            usedClusterID = mixed
        elif p == 1:
            usedClusterID = rural
        elif p == 2:
            usedClusterID = urban

        if type == 0:
            temp = calSeperateTPFPFN(dataMatrix[:,1], usedClusterID, listID, riskarea_pop[p])
        elif type == 1:
            temp = calSeperateTPFPFN(dataMatrix[:, repeat + 2], usedClusterID, listID, riskarea_dis[p])
        elif type == 2:
            temp = calSeperateTPFPFN(dataCount, usedClusterID, listID, len(usedClusterID))
        #temp = calSeperateTPFPFN(dataMatrix[:,1], usedClusterID, listID, riskarea_pop[p])
        #temp = calSeperateTPFPFN(dataMatrix[:, repeat + 2], usedClusterID, listID, riskarea_dis[p])
        #temp = calSeperateTPFPFN(dataCount, usedClusterID, listID, len(usedClusterID))
        if temp[-1] > win_score[-1]:
            win_score = temp
            win_p = p
    return win_score, win_p

#--------------------------------------------------------------------------
#MAIN
if __name__ == "__main__":
    print "begin at " + getCurTime()
    mixed = [91,98,101,104,114,115,119,126,131,142,146,147,154,162,168,172]
    rural = [8,9,10,11,12,13,14,15,17,19,20,26,28,33,34,37]
    urban = [105,107,112,120,122,125,127,128,130,133,134,141,143,149,152,155]

    hot_1 = [9,130,147]
    hot_2 = hot_1 + [10,133,154]
    hot_4 = hot_2 + [12,17,125,131,141,146]
    hot_8 = hot_4 + [14,19,20,26,114,115,119,120,128,134,149,168]
    hot_16 = mixed + rural + urban

    dataused = hot_16
    if len(dataused)/3 < 10:
        unitCSV = 'C:/_DATA/CancerData/SatScan/mult6000/three0' + str(len(dataused)/3) + '_format.csv'
    else:
        unitCSV = 'C:/_DATA/CancerData/SatScan/mult6000/three' + str(len(dataused)/3) + '_format.csv'
            
    dataMatrix = build_data_list(unitCSV)  # [id, pop, cancer1, cancer2, cancer3]
    riskarea_pop = calSum(dataMatrix[:,1])  # [mixed, rural, urban, total]

    type = 0 # 0: pop, 1: cancer, 2: count
    saveOutput = []
    for significanceLevel in range(1, 20):
        significanceLevel = 0.05 * significanceLevel

        dataCount = np.ones(len(dataMatrix))
        power = 0
        power2 = 0
        power3 = 0
        output = np.array([])
        scoreID = np.zeros((1000,3))
        for i in range(0,1000):
            for j in range(0,3):
                scoreID[i, j] = -1
        #print scoreID
        row = 0
        count = 1
        for repeat in range(0, 1000):
            #row += 1
            #if int(row/100) > 0:
                #print count*100
                #row = 0
                #count += 1
            riskarea_dis = calSum(dataMatrix[:, repeat + 2])    #[mixed, rural, urban, total]
            
            if len(dataused)/3 < 10:
                satscanCSV = 'C:/_DATA/CancerData/SatScan/mult6000/three0'
                satscanCSV += str(len(dataused)/3) + '/' + str(repeat) + '.gis.dbf'
            else:
                satscanCSV = 'C:/_DATA/CancerData/SatScan/mult6000/three'
                satscanCSV += str(len(dataused)/3) + '/' + str(repeat) + '.gis.dbf'
                
            #satscanCSV = 'C:/_DATA/CancerData/test/Jan15/satscan/highlow/' + str(repeat) + '.dbf'
            #satscan_id = build_satscanresult_dbf(satscanCSV)
            # generate [unitID, regionID, p_value]
            #unitAttri = []    # all units
            unitPvalue = build_satscanresult_dbf(satscanCSV)   # units with p_value < 0.05
            cluster1 = []
            cluster2 = []
            cluster3 = []
            for item in unitPvalue:
                if int(item[1]) == 1:
                    cluster1.append(int(item[0]))
                if int(item[1]) == 2:
                    cluster2.append(int(item[0]))
                if int(item[1]) == 3:
                    cluster3.append(int(item[0]))
            flag = 0
            total_result = np.zeros(4)
            detectedArea = [-1,-1,-1]
            for p in range(0,3):
                if p == 0:
                    usedClusterID = cluster1
                elif p == 1:
                    usedClusterID = cluster2
                elif p == 2:
                    usedClusterID = cluster3
                    
                if len(usedClusterID) > 0:
                    flag = 1
                    power += 1
                    temp_result, scoreID[repeat, p] = findHighestScore(usedClusterID)
                    if scoreID[repeat, p] > -1:
                        power2 += 1
                        if scoreID[repeat, p] not in detectedArea:
                            detectedArea[int(scoreID[repeat, p])] = scoreID[repeat, p]
                            power3 += 1
                    for i in range(0,3):
                        total_result[i] += temp_result[i]
            
                
    ##        if len(unitPvalue) > 0:
    ##            unitAttri = np.array(unitAttri)
    ##            unitPvalue = np.array(unitPvalue)
    ##        else:
    ##            continue
    ##        
    ##        total_result = np.zeros(12)
    ##        for p in range(0,3):
    ##            if p == 0:
    ##                usedClusterID = mixed
    ##            elif p == 1:
    ##                usedClusterID = rural
    ##            elif p == 2:
    ##                usedClusterID = urban
    ##            regionID = findRegionID(unitPvalue, usedClusterID)
    ##            #regionID = findRegionID(unitPvalue, rural)
    ##            #regionID = findRegionID(unitPvalue, urban)
    ##            #print mixedID, ruralID, urbanID
    ##            #print regionID
    ##            if len(regionID) > 0:
    ##                power += 1
    ##                listID = findUnitIDforRegionID(unitPvalue, regionID)
    ##                #print listID
    ##            else:
    ##                continue
    ##            
    ##            temp_result = calTPFPFN(dataMatrix, usedClusterID, listID, riskarea_pop[p], riskarea_dis[p])
    ##            
    ##            for i in range(0,9):
    ##                total_result[i] += temp_result[i]
    ##            #print total_result
    ##
    ##            if p == 0:
    ##                mixed_output = np.append(mixed_output, temp_result)
    ##            elif p == 1:
    ##                rural_output = np.append(rural_output, temp_result)
    ##            elif p == 2:
    ##                urban_output = np.append(urban_output, temp_result)
            temp_value = total_result[0] + total_result[1] + total_result[2]
            if flag > 0:
                total_result[3] = total_result[0]/(total_result[0] + total_result[1] + total_result[2])
                output = np.append(output, total_result)
        #print output
        filePath = 'C:/_DATA/CancerData/SatScan/mult6000/satscan' + str(len(dataused)/3) + '_newCTG'
    ##    mixed_output.shape = (-1, 12)
    ##    np.savetxt(filePath + '_mixed.csv', mixed_output, delimiter=',', fmt = '%10.5f')
    ##    rural_output.shape = (-1, 12)
    ##    np.savetxt(filePath + '_rural.csv', rural_output, delimiter=',', fmt = '%10.5f')
    ##    urban_output.shape = (-1, 12)
    ##    np.savetxt(filePath + '_urban.csv', urban_output, delimiter=',', fmt = '%10.5f')
        output.shape = (-1, 4)
        #np.savetxt(filePath + '_pop.csv', output, delimiter=',', fmt = '%10.5f')
        print '------------------------'
        print significanceLevel
        print np.average(output[:,3])
        #np.savetxt(filePath + '_scoreID.csv', scoreID, delimiter=',', fmt = '%10.5f')
        
        print 'power = ' + str(power)
        print 'power2 = ', power2
        print 'power3 = ', power3
        saveOutput.append([np.average(output[:,3]), power, power2, power3])
    np.savetxt(satscanCSV[:-4] + str(type) + '_power.csv', saveOutput, delimiter=',', fmt = '%10.5f')
    print "end at " + getCurTime()
    print "========================================================================"  
    
            